{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# neural network of many complex valued neurons in 3 layers\n",
    "# applied to the MNIST dataset\n",
    "import numpy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "class neuralNetwork:\n",
    "    \n",
    "    def __init__(self, inputnodes, hiddennodes, outputnodes, cats, periods):\n",
    "        # set number of nodes in each input, hidden, output layer\n",
    "        self.inodes = inputnodes +1 # increament for bias node\n",
    "        self.hnodes = hiddennodes\n",
    "        self.onodes = outputnodes\n",
    "        \n",
    "        # link weight matrices, wih and who\n",
    "        # weights inside the arrays are w_i_j, where link is from node i to node j in the next layer\n",
    "        # w11 w21\n",
    "        # w12 w22 etc \n",
    "        self.wih = numpy.random.uniform(-1.0, 1.0, (self.hnodes, self.inodes))\n",
    "        self.wih = numpy.array(self.wih, ndmin=2, dtype='complex128')\n",
    "        self.wih += 1j * numpy.random.uniform(-1.0, 1.0, (self.hnodes, self.inodes))\n",
    "        \n",
    "        self.who = numpy.random.uniform(-1.0, 1.0, (self.onodes, self.hnodes))\n",
    "        self.who = numpy.array(self.who, ndmin=2, dtype='complex128')\n",
    "        self.who += 1j * numpy.random.random((self.onodes, self.hnodes))\n",
    "        \n",
    "        # number of output class categories\n",
    "        self.categories = cats\n",
    "        \n",
    "        # todo periodicity\n",
    "        self.periodicity = periods\n",
    "        pass\n",
    "    \n",
    "    def z_to_class(self, z):\n",
    "        # first work out the angle, but shift angle from [-pi/2, +pi.2] to [0,2pi]\n",
    "        angle = numpy.mod(numpy.angle(z) + 2*numpy.pi, 2*numpy.pi)\n",
    "        # from angle to category\n",
    "        p = int(numpy.floor (self.categories * angle / (2*numpy.pi)))\n",
    "        return p\n",
    "\n",
    "    def class_to_angle(self, p):\n",
    "        # class to angle, using bisector\n",
    "        angle = ((p + 0.5) / self.categories) * 2 * numpy.pi\n",
    "        return angle\n",
    "    \n",
    "    def status(self):\n",
    "        print (\"self.wih = \", self.wih)\n",
    "        print (\"self.who = \", self.who)\n",
    "        pass\n",
    "\n",
    "    def query(self, inputs_list):\n",
    "        # add bias input\n",
    "        inputs_list.append(1.0)\n",
    "        \n",
    "        # convert input to complex\n",
    "        inputs = numpy.array(inputs_list, ndmin=2, dtype='complex128').T\n",
    "        #print(\"inputs = \\n\", inputs)\n",
    "        \n",
    "        # signal into hidden layer\n",
    "        hidden_inputs = numpy.dot(self.wih, inputs)\n",
    "        #print(\"hidden_inputs = \", hidden_inputs)\n",
    "        #signal out of hidden layer\n",
    "        hidden_outputs = numpy.exp(1j * numpy.angle(hidden_inputs))\n",
    "        #print(\"hidden_outputs = \", hidden_outputs)\n",
    "        \n",
    "        # signal into final output layer\n",
    "        final_inputs = numpy.dot(self.who, hidden_outputs)\n",
    "        #print(\"final_input = \", final_inputs)\n",
    "        #signal out of output layer\n",
    "        final_outputs = numpy.exp(1j * numpy.angle(final_inputs))\n",
    "        #print(\"final_outputs = \", final_outputs)\n",
    "        \n",
    "        # map to output classes\n",
    "        output_classes = self.z_to_class(final_outputs)\n",
    "        #print(\"output_classes = \", output_classes)\n",
    "        return output_classes\n",
    "    \n",
    "    def train(self, inputs_list, target_class_list):\n",
    "        # add bias input\n",
    "        inputs_list.append(1.0)\n",
    "        \n",
    "        # convert input to complex\n",
    "        inputs = numpy.array(inputs_list, ndmin=2, dtype='complex128').T\n",
    "        #print(\"inputs = \\n\", inputs)\n",
    "        \n",
    "        # map target classes to unit circle\n",
    "        targets = numpy.exp(1j * self.class_to_angle(numpy.array(target_class_list, ndmin=2).T))\n",
    "        #print(\"targets = \\n\", targets)\n",
    "\n",
    "        # signal into hidden layer\n",
    "        hidden_inputs = numpy.dot(self.wih, inputs)\n",
    "        #print(\"hidden_inputs = \", hidden_inputs)\n",
    "        #signal out of hidden layer\n",
    "        hidden_outputs = numpy.exp(1j * numpy.angle(hidden_inputs))\n",
    "        #print(\"hidden_outputs = \", hidden_outputs)\n",
    "        \n",
    "        # signal into final output layer\n",
    "        final_inputs = numpy.dot(self.who, hidden_outputs)\n",
    "        #print(\"final_inputs = \", final_inputs)\n",
    "        #signal out of output layer\n",
    "        final_outputs = numpy.exp(1j * numpy.angle(final_inputs))\n",
    "        #print(\"final_outputs = \", final_outputs)\n",
    "        \n",
    "        # output layer error is the (target - actual)\n",
    "        output_errors = targets - final_outputs\n",
    "        #print(\"output_errors = \", output_errors)\n",
    "        # hidden layer error is the output_errors split simply (not by weights)\n",
    "        hidden_errors = numpy.dot(numpy.array(numpy.ones((self.hnodes)), ndmin=2).T / self.hnodes, output_errors)\n",
    "        #print(\"hidden_errors = \", hidden_errors)\n",
    "        \n",
    "        # dw = e * x.T / (x.x.T)\n",
    "        dwho = numpy.dot(output_errors, numpy.conj(hidden_outputs.T)) / (self.hnodes)\n",
    "        #print(\"dwho = \", dwho)\n",
    "        self.who += dwho\n",
    "        \n",
    "        dwih = numpy.dot(hidden_errors, numpy.conj(inputs.T)) / (self.inodes)\n",
    "        #print(\"dwih = \", dwih)\n",
    "        self.wih += dwih        \n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# number of input, hidden and output nodes\n",
    "input_nodes = 784\n",
    "hidden_nodes = 100\n",
    "output_nodes = 1\n",
    "\n",
    "# categories, periodicity\n",
    "categories = 10\n",
    "periodicity = 1\n",
    "\n",
    "# create instance of neural network\n",
    "n = neuralNetwork(input_nodes,hidden_nodes,output_nodes, categories, periodicity)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load the mnist training data CSV file into a list\n",
    "training_data_file = open(\"mnist_dataset/mnist_train.csv\", 'r')\n",
    "training_data_list = training_data_file.readlines()\n",
    "training_data_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# train neural network - OR\n",
    "\n",
    "epochs = 1\n",
    "\n",
    "for e in range(epochs):\n",
    "    # go through all records in the training data set\n",
    "    for record in training_data_list:\n",
    "        # split the record by the ',' commas\n",
    "        all_values = record.split(',')\n",
    "        # scale and shift the inputs\n",
    "        inputs = numpy.exp(1j * (numpy.asfarray(all_values[1:]) / 255.0) * numpy.pi)\n",
    "        inputs_list = inputs.tolist()\n",
    "        # create the target class list\n",
    "        targets = [int(all_values[0])]\n",
    "        n.train(inputs_list, targets)\n",
    "        pass\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load the mnist test data CSV file into a list\n",
    "test_data_file = open(\"mnist_dataset/mnist_test.csv\", 'r')\n",
    "test_data_list = test_data_file.readlines()\n",
    "test_data_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# test the neural network\n",
    "\n",
    "# scorecard for how well the network performs, initially empty\n",
    "scorecard = []\n",
    "\n",
    "# go through all the records in the test data set\n",
    "for record in test_data_list:\n",
    "    # split the record by the ',' commas\n",
    "    all_values = record.split(',')\n",
    "    # correct answer is first value\n",
    "    correct_label = int(all_values[0])\n",
    "    # scale and shift the inputs\n",
    "    inputs = numpy.exp(1j * (numpy.asfarray(all_values[1:]) / 255.0) * numpy.pi)\n",
    "    inputs_list = inputs.tolist()\n",
    "    # query the network\n",
    "    output_label = n.query(inputs_list)\n",
    "    # append correct or incorrect to list\n",
    "    if (output_label == correct_label):\n",
    "        # network's answer matches correct answer, add 1 to scorecard\n",
    "        scorecard.append(1)\n",
    "    else:\n",
    "        # network's answer doesn't match correct answer, add 0 to scorecard\n",
    "        scorecard.append(0)\n",
    "        pass\n",
    "    \n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "performance =  0.1719\n"
     ]
    }
   ],
   "source": [
    "# calculate the performance score, the fraction of correct answers\n",
    "scorecard_array = numpy.asarray(scorecard)\n",
    "print (\"performance = \", scorecard_array.sum() / scorecard_array.size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
